"""
# 处理大型分类变量：特征散列化
"""

import pandas as pd
import json
from sklearn.feature_extraction import FeatureHasher
from sys import getsizeof

# 加载前10 000条点评
with open('../数据集/yelp_academic_dataset_review.json') as f:
    js = [json.loads(x) for x in f.readlines()]
review_df = pd.DataFrame(js[:10000])
# 定义m为唯一的business_id的数量
m = len(review_df.business_id.unique())
print(m)
h = FeatureHasher(n_features=m, input_type='string')
f = h.transform(review_df['business_id'])

# 散列化对特征可解释性的影响
print(review_df['business_id'].unique().tolist()[0:5])
print(f.toarray())

# 不是很好，但是看一下特征的存储空间大小
print('Our pandas Series, in bytes: ', getsizeof(review_df['business_id']))
print('Our hashed numpy array, in bytes: ', getsizeof(f))
